{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/guanyu/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-927b70275c58>:8: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/guanyu/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/guanyu/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/guanyu/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/guanyu/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/guanyu/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf  \n",
    "from tensorflow.examples.tutorials.mnist import input_data  \n",
    "  \n",
    "  \n",
    "#载入数据集  \n",
    "#当前路径  \n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置后面用到的批次大小"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "550\n"
     ]
    }
   ],
   "source": [
    "#每个批次的大小\n",
    "#以矩阵的形式放进去\n",
    "batch_size = 100\n",
    "#计算一共有多少个批次\n",
    "n_batch = mnist.train.num_examples // batch_size ## 地板除 - 操作数的除法，其结果是删除小数点后的商数。 但如果其中一个操作数为负数，则结果将被保留，即从零(向负无穷大)舍去\n",
    "print(n_batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 增加隐层，确定模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y = tf.placeholder(tf.float32, [None, 10])\n",
    "#学习率\n",
    "lr = tf.Variable(0.001, dtype=tf.float32)\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "in_units=784 #输入节点数\n",
    "h1_units=500#隐含层的输出节点数,这里试验过300，400，100\n",
    "h2_units = 300 #第二隐含层的节点数\n",
    "\n",
    "# w1= tf.Variable(tf.truncated_normal([in_units,h1_units],stddev=0.1))\n",
    "w1= tf.Variable(tf.contrib.layers.xavier_initializer()((in_units,h1_units)))\n",
    "b1=tf.Variable(tf.zeros([h1_units])+0.1)\n",
    "# w2=tf.Variable(tf.truncated_normal([h1_units,h2_units],stddev=0.1))\n",
    "w2= tf.Variable(tf.contrib.layers.xavier_initializer()((h1_units,h2_units)))\n",
    "\n",
    "b2=tf.Variable(tf.zeros(h2_units)+0.1)\n",
    "# w3=tf.Variable(tf.zeros([h2_units,10]))\n",
    "w3=tf.Variable(tf.truncated_normal([h2_units,10],stddev=0.1))\n",
    "# w3=tf.Variable(np.random.randn(h2_units,10)/np.sqrt(h2_units/2))\n",
    "b3=tf.Variable(tf.zeros([10])+0.1)\n",
    "\n",
    "x=tf.placeholder(tf.float32,[None,in_units])\n",
    "hidden1=tf.nn.relu(tf.matmul(x,w1)+b1)  \n",
    "L1_drop = tf.nn.dropout(hidden1,keep_prob)  \n",
    "\n",
    "hidden2=tf.nn.relu(tf.matmul(L1_drop,w2)+b2)  \n",
    "L2_drop = tf.nn.dropout(hidden2,keep_prob)  \n",
    "\n",
    "prediction=tf.nn.softmax(tf.matmul(L2_drop,w3)+b3)  #在输出层依然使用的softmax分类\n",
    "\n",
    "## 使用L2正则\n",
    "\n",
    "tf.add_to_collection(tf.GraphKeys.WEIGHTS, w1)\n",
    "tf.add_to_collection(tf.GraphKeys.WEIGHTS, w2)\n",
    "tf.add_to_collection(tf.GraphKeys.WEIGHTS, w3)\n",
    "regularizer = tf.contrib.layers.l2_regularizer(5/50000)## 参数设为0.00001\n",
    "reg_term = tf.contrib.layers.apply_regularization(regularizer)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置交叉熵函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#交叉熵代价函数\n",
    "# loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=prediction))+reg_term\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=prediction))+reg_term"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#训练\n",
    "train_step = tf.train.AdamOptimizer(lr).minimize(loss)\n",
    "# train_step = tf.train.GradientDescentOptimizer(lr).minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#初始化变量\n",
    "init = tf.global_variables_initializer()\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(prediction, 1))\n",
    "#求准确率\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 运用批次，训练，不断迭代，达到学习率衰减，然后打印出准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "周期：0, 准确率：0.953, 学习率：0.001\n",
      "周期：1, 准确率：0.9653, 学习率：0.00095\n",
      "周期：2, 准确率：0.9696, 学习率：0.0009025\n",
      "周期：3, 准确率：0.9735, 学习率：0.000857375\n",
      "周期：4, 准确率：0.9762, 学习率：0.00081450626\n",
      "周期：5, 准确率：0.9761, 学习率：0.0007737809\n",
      "周期：6, 准确率：0.9778, 学习率：0.0007350919\n",
      "周期：7, 准确率：0.9769, 学习率：0.0006983373\n",
      "周期：8, 准确率：0.9791, 学习率：0.0006634204\n",
      "周期：9, 准确率：0.9757, 学习率：0.0006302494\n",
      "周期：10, 准确率：0.9762, 学习率：0.0005987369\n",
      "周期：11, 准确率：0.9801, 学习率：0.0005688001\n",
      "周期：12, 准确率：0.9815, 学习率：0.0005403601\n",
      "周期：13, 准确率：0.9811, 学习率：0.0005133421\n",
      "周期：14, 准确率：0.9802, 学习率：0.000487675\n",
      "周期：15, 准确率：0.9789, 学习率：0.00046329122\n",
      "周期：16, 准确率：0.9821, 学习率：0.00044012666\n",
      "周期：17, 准确率：0.9817, 学习率：0.00041812033\n",
      "周期：18, 准确率：0.9807, 学习率：0.00039721432\n",
      "周期：19, 准确率：0.9817, 学习率：0.0003773536\n",
      "周期：20, 准确率：0.9815, 学习率：0.00035848594\n",
      "周期：21, 准确率：0.9823, 学习率：0.00034056162\n",
      "周期：22, 准确率：0.9831, 学习率：0.00032353355\n",
      "周期：23, 准确率：0.9828, 学习率：0.00030735688\n",
      "周期：24, 准确率：0.9825, 学习率：0.000291989\n",
      "周期：25, 准确率：0.9843, 学习率：0.00027738957\n",
      "周期：26, 准确率：0.982, 学习率：0.0002635201\n",
      "周期：27, 准确率：0.9831, 学习率：0.00025034408\n",
      "周期：28, 准确率：0.9832, 学习率：0.00023782688\n",
      "周期：29, 准确率：0.9839, 学习率：0.00022593554\n",
      "周期：30, 准确率：0.9836, 学习率：0.00021463877\n",
      "周期：31, 准确率：0.9836, 学习率：0.00020390682\n",
      "周期：32, 准确率：0.9832, 学习率：0.00019371149\n",
      "周期：33, 准确率：0.9823, 学习率：0.0001840259\n",
      "周期：34, 准确率：0.9837, 学习率：0.00017482461\n",
      "周期：35, 准确率：0.9837, 学习率：0.00016608338\n",
      "周期：36, 准确率：0.9827, 学习率：0.00015777921\n",
      "周期：37, 准确率：0.9805, 学习率：0.00014989026\n",
      "周期：38, 准确率：0.9826, 学习率：0.00014239574\n",
      "周期：39, 准确率：0.9843, 学习率：0.00013527596\n",
      "周期：40, 准确率：0.9837, 学习率：0.00012851215\n",
      "周期：41, 准确率：0.9831, 学习率：0.00012208655\n",
      "周期：42, 准确率：0.9846, 学习率：0.00011598222\n",
      "周期：43, 准确率：0.9855, 学习率：0.00011018311\n",
      "周期：44, 准确率：0.9837, 学习率：0.000104673956\n",
      "周期：45, 准确率：0.9834, 学习率：9.944026e-05\n",
      "周期：46, 准确率：0.9838, 学习率：9.446825e-05\n",
      "周期：47, 准确率：0.9831, 学习率：8.974483e-05\n",
      "周期：48, 准确率：0.9834, 学习率：8.525759e-05\n",
      "周期：49, 准确率：0.9846, 学习率：8.099471e-05\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    #总共50个周期\n",
    "    for epoch in range(50):\n",
    "        #刚开始学习率比较大，后来慢慢变小\n",
    "        sess.run(tf.assign(lr, 0.001 * (0.95 ** epoch)))\n",
    "        #总共n_batch个批次\n",
    "        for batch in range(n_batch):\n",
    "            #获得一个批次\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_step, feed_dict={x:batch_xs, y:batch_ys, keep_prob:1.0})\n",
    "        \n",
    "        learning_rate = sess.run(lr)\n",
    "        #训练完一个周期后测试数据准确率\n",
    "        acc = sess.run(accuracy, feed_dict={x:mnist.test.images, y:mnist.test.labels, keep_prob:1.0})\n",
    "        \n",
    "        print(\"周期：\" + str(epoch) + \", 准确率：\" + str(acc)+ \", 学习率：\" + str(learning_rate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结：\n",
    "## 1 尝试了单隐层，结果最后没有超过0.95\n",
    "## 2 尝试了双隐层，最后结果大概是0.956\n",
    "## 3 更换激活函数，从sigmoid变成relu，结果提高到了0.976\n",
    "## 4 增加了L2正则，结果变成了0.8761\n",
    "## 5 增加了训练的循环次数，结果变成了0.9636\n",
    "## 6 增加了隐层神经元的数量，结果增加到了0.9723\n",
    "## 7 改进了权重的初始化，（Xavier）结果增加到了0.973\n",
    "## 8 设置学习率衰减，最终突破了0.98\n",
    "## 9 如果使用tf.nn.sigmoid_cross_entropy_with_logits+Adam则最终结果无法突破0.977（无法在50次周期循环之内达到）\n",
    "## 10 GradientDescentOptimizer无法在当前设置的初始值，在50次循环下突破0.72"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
